/********************************************************************
  Copyright by TAOBAO
  File Name: dbloader.cpp
  Created Date: 2009-05-09
  Author: ALI
  Version: 0.0.1
  Last Version Date:
  Version Date:
  Description: Abstract class for database loader
  Modified history:
  Date      Author    Description
--------------------------------------------------------------------
  20090509  ALI       Abstract class for database loader (create)
  20100706  ALI       Fix QCCheck when files are empty
********************************************************************/
#include "dbloader.h"

const char DBLoader::m_achSeps[] = {',', 0x01, '"', '\t', ' ', '~', '^' };
//Constructor
DBLoader::DBLoader()
{
	// load result initial
	m_nReadRows = 0;
	m_nSkipRows = 0;
	m_nLoadRows = 0;
	m_nDeleteRows = 0;
	m_nRejRows = 0;
	m_nCommitRows = 0;
}

DBLoader::~DBLoader()
{
}

bool DBLoader::GuessInfo( char &chSep, int &nColNum )
{
    const int GUESSLINE = 100;  // lines used to guess delimiter
    const int MAXCHAR = 256;     // max char value
    int nHints[MAXCHAR-1] = {0}; // mismatch hints
    char szLine[8192];

    vector<string>::iterator iter = m_vFiles.begin();
    CIFile cFile;
    if( !cFile.Open( iter->c_str(), "r" ) )
        return false;

    int pContext[GUESSLINE][MAXCHAR-1];
    memset(pContext, 0x00, GUESSLINE*(MAXCHAR-1)*sizeof(int));
    // 得到256个字符在每行中出现的次数
    int nRows = 0;
    for( nRows = 0; nRows < GUESSLINE; nRows++ )
    {
        memset( szLine, 0x00, sizeof(szLine) );
        if( cFile.GetLine(szLine, sizeof(szLine)) )
        {
            for( int j = 0; j < strlen(szLine); j++)
                pContext[nRows][szLine[j]]++;
        }
        else if( ++iter != m_vFiles.end() )  // still have files to read
        {
            cFile.Close();
            nRows--;          // fix row number
            if( !cFile.Open( iter->c_str(), "r" ) )
            {
//                delete []pContext;
                return false;
            }
        }
        else
        {
            cFile.Close();
            break;
        }
    }
    cFile.Close();

    // 
    int nMinHint = GUESSLINE;
    for( int i = 0; i < sizeof(m_achSeps) && m_achSeps[i]; i++ )
    {
        map<int,int> mWordFreq;          // 记录词频
        int nSep = m_achSeps[i];
        int nLastCount = pContext[0][nSep];
        for( int j = 0; j < nRows; j++ )
        {
            if( nLastCount != pContext[j][nSep] )
            {
                nHints[i]++;
                nLastCount = pContext[j][nSep];
            }
            mWordFreq[nLastCount]++;
        }
        if( nHints[i] < nMinHint )  // 选取归一化程度最高的字符
        {
            int nMax = -1; 
            int nCols = 0;
            for( map<int,int>::iterator iter = mWordFreq.begin(); iter != mWordFreq.end(); iter++ ) // 出现次数最多的字段数
            {
                if( iter->second > nMax )
                {
                    nCols = iter->first;
                    nMax = iter->second;
                }
            }
            if( nCols == 0 )        // 字段数为0出现最多,忽略
                continue;

            nMinHint = nHints[i];
            chSep = m_achSeps[i];   // 记录猜测到的分隔符
            nColNum = nCols + 1;    // 分隔符出现次数加1为字段数
        } 
    }

//    delete []pContext;
    // 如果不归一数达到千分之五~十,或者文件行数少于指定行数则认为猜测失败
    if( nMinHint > GUESSLINE * 0.01 || nRows < GUESSLINE )
        return false;

    return true;
}

bool DBLoader::QCCheck()
{
    if( 0 == m_nReadRows ) // empty file, ignore limit and check ok, 20100706
        return true;
    // Quality control
    if( m_fLimit < 1e-6 && m_fLimit > -1e-6)     // limit = 0
    { // limit = 0
        return (0 == m_nRejRows); /* finish the load */
    }
    else if( m_fLimit <= 1 )
    { // percent
        if( m_nRejRows/(float)m_nReadRows <= m_fLimit )
            return true;
        else
            return false;
    }
    else
    { // abs rows
        if( m_nRejRows <= m_fLimit )
            return true;
        else
            return false;
    }
}

void DBLoader::GetLoadResult( int *pResult )
{
	pResult[0] = m_nReadRows;
	pResult[1] = m_nSkipRows;
	pResult[2] = m_nLoadRows;
	pResult[3] = m_nDeleteRows;
	pResult[4] = m_nRejRows;
	pResult[5] = m_nCommitRows;
}


